import pandas as pd
import datetime
from utils.read_url import get_url
from utils.trans_date import transale_date
from config import config


def _parse_licai(url):
    arr = []
    html_soup = get_url(url)
    if html_soup:
        soup_list = html_soup.find('tbody').find_all('tr') if html_soup.find('tbody') else None
        for soup in soup_list:
            data_list = soup.find_all('td')
            if data_list:
                mid_arr = []
                for info in data_list:
                    data = info.get_text().replace('年', '/').replace('月', '/').replace('-', '/').replace('日', '').replace('%', '')
                    if data:
                        mid_arr.append(data)
                if '个人' in mid_arr[-1]:
                    arr.append(mid_arr)

        return arr


def read_licai_page(page_num):
    arr = []
    url = config.LICAI_URL
    if page_num > 1:
        url_page = config.LICAI_PAGE_URL.format(page_num - 1)
        url = '{}{}'.format(url, url_page)

    html_soup = get_url(url)
    if html_soup:
        soup_data = html_soup.find('div', class_='news')
        if soup_data:
            for data in soup_data.find_all('li'):
                date = data.find('span').get_text()
                date = date.replace('[', '').replace(']', '').strip() if date else None
                title = data.find('a').get('title')
                if '销售公告' in title:
                    info_url = data.find('a').get('href')[2:]
                    info_url = '{}{}'.format(config.LICAI_URL, info_url)
                    mid_arr = _parse_licai(url=info_url)
                    if mid_arr:
                        arr += mid_arr

    df = pd.DataFrame(arr)
    if not df.empty:
        if len(arr) == 1:
            df = df.T
        df.columns = ['code', 'sale_begin', 'sale_end', 'benifit_begin', 'end', 'periods', 'pre_benifit', 'sale_object']
        df['sale_end'] = df['sale_end'].apply(transale_date)
        # sale for today
        df = df[df['sale_end'] >= datetime.datetime.today().date()]

    return df


def read_licai():
    df = pd.DataFrame()
    i = 1
    while True:
        print(i)
        mid_df = read_licai_page(page_num = i)
        if not mid_df.empty:
            df = pd.concat([df, mid_df])
            i += 1
        else:
            break
    return df.sort_values('pre_benifit', ascending=False)


def read_licai_today():
    today = datetime.datetime.now().strftime('%Y/%m/%d')
    today_year = today[:5]
    today_month_day = today[5:]
    today = '{}{}'.format(today_year, today_month_day.replace('0', ''))
    df = pd.DataFrame()
    i = 1
    while True:
        print(i)
        mid_df = read_licai_page(page_num = i)
        mid_df = mid_df[mid_df['sale_begin'] == today]
        if not mid_df.empty:
            df = pd.concat([df, mid_df])
            i += 1
        else:
            break
    if not df.empty:
        df = df.sort_values('pre_benifit', ascending=False)
    return df


df = read_licai_today()
print(df)
